In [2]:
%pylab inline


Populating the interactive namespace from numpy and matplotlib

In [3]:
import pandas
import matplotlib.pyplot as plt
import numpy as np
import sunpy
import matplotlib
import seaborn as sns
from scipy import stats


/Users/schriste/anaconda/lib/python2.7/site-packages/pytz/__init__.py:35: UserWarning: Module _imaging was already imported from /Users/schriste/anaconda/lib/python2.7/site-packages/PIL/_imaging.so, but /Users/schriste/.local/lib/python2.7/site-packages is being added to sys.path
  from pkg_resources import resource_stream

In [4]:
sns.set_color_palette("deep", desat=.6)

In [5]:
import heroespy

First load the data, this loads all of the data from all the solar observations, these files can be recreated using the script inside notebooks called


In [6]:
date_format = "%Y-%m-%d %H:%M:%S.%f"

In [77]:
file = "/Users/schriste/Dropbox/Developer/HEROES/HEROES-Telescope/SAS1_pointing_data.csv"

In [78]:
data = pandas.read_csv(file, parse_dates=True, index_col = 0)

In [79]:
data


WARNING: DeprecationWarning: height has been deprecated.
 [pandas.core.config]
Out[79]:
<class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 103820 entries, 2013-09-21 15:21:04.896474 to 2013-09-21 22:35:54.476023
Data columns (total 7 columns):
ctl az        103820  non-null values
ctl el        103820  non-null values
offset r      103820  non-null values
offset x      103820  non-null values
offset y      103820  non-null values
pointing x    103820  non-null values
pointing y    103820  non-null values
dtypes: float64(7)

In [9]:
data.describe()


WARNING: DeprecationWarning: height has been deprecated.
 [pandas.core.config]
WARNING: DeprecationWarning: height has been deprecated.
 [pandas.core.config]
Out[9]:
ctl az ctl el offset r offset x offset y pointing x pointing y
count 103820.000000 103820.000000 103820.000000 103820.000000 103820.000000 103820.000000 103820.000000
mean 456.845001 450.987216 35.810597 13.521916 -0.544368 -758.352848 98.613347
std 301.608637 320.446334 55.991746 55.375291 34.174898 139.655021 37.704426
min -2296.241712 -265.489180 0.153758 -2375.628042 -243.421328 -3169.628042 -153.527507
25% 147.645012 126.814234 17.269976 -4.666385 -13.800850 -797.662825 87.068193
50% 517.894013 546.877001 28.427181 12.031613 -0.646516 -780.940290 100.658258
75% 757.390250 754.090286 43.739924 28.787960 12.413126 -763.711287 113.737244
max 975.925274 3428.824061 3478.211166 794.000000 2540.540163 123.294351 2642.540163

The large max values suggest that we have some bad data which we should get rid of


In [10]:
5*data['offset x'].std()


Out[10]:
276.87645267998181

In [11]:
con1 = np.abs(data['offset x']) < 5*data['offset x'].std()
con2 = np.abs(data['offset y']) < 5*data['offset y'].std()
index = (con1 * con2) == 1

In [12]:
np.sum(index)


Out[12]:
103429

Number of bad values


In [13]:
np.sum(index == False)


Out[13]:
391

In [14]:
clean_data = data[index]
clean_data.describe()


WARNING: DeprecationWarning: height has been deprecated.
 [pandas.core.config]
WARNING: DeprecationWarning: height has been deprecated.
 [pandas.core.config]
Out[14]:
ctl az ctl el offset r offset x offset y pointing x pointing y
count 103429.000000 103429.000000 103429.000000 103429.000000 103429.000000 103429.000000 103429.000000
mean 458.589888 452.236000 32.964657 11.307194 -0.497031 -760.529989 98.655857
std 299.966732 318.739132 21.841575 27.611489 25.946835 131.630390 30.490684
min -166.261230 -251.100155 0.153758 -181.921719 -170.141565 -975.921719 -153.527507
25% 149.908004 130.195563 17.232539 -4.726709 -13.653196 -797.716215 87.218421
50% 519.877876 549.583433 28.339230 11.927918 -0.579880 -781.042553 100.724322
75% 757.651467 754.232165 43.521073 28.606733 12.442659 -763.918401 113.763850
max 975.925274 971.008541 286.971154 256.454461 169.227704 123.294351 267.135965

In [16]:
col = 'ctl az'
fig = plt.figure(figsize=(8,6), dpi=100)
clean_data[col].plot()


Out[16]:
<matplotlib.axes.AxesSubplot at 0x10c041c10>

The CTL az should not be changing as a function of time! It should be in sky coordinates and following the Sun! Something is wrong here.

A close up view of the time series


In [17]:
fig = plt.figure(figsize=(8,6), dpi=100)
clean_data[col][10000:12000].plot()


Out[17]:
<matplotlib.axes.AxesSubplot at 0x109971e90>

In [81]:
from scipy import fftpack

In [80]:
clean_data.resample('10s')
y = clean_data


WARNING: DeprecationWarning: height has been deprecated.
 [pandas.core.config]
Out[80]:
<class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 2602 entries, 2013-09-21 15:21:00 to 2013-09-21 22:34:30
Freq: 10S
Data columns (total 7 columns):
ctl az        2602  non-null values
ctl el        2602  non-null values
offset r      2602  non-null values
offset x      2602  non-null values
offset y      2602  non-null values
pointing x    2602  non-null values
pointing y    2602  non-null values
dtypes: float64(7)

In [18]:
col = 'ctl el'
fig = plt.figure(figsize=(8,6), dpi=100)
clean_data[col].plot()


Out[18]:
<matplotlib.axes.AxesSubplot at 0x10963e750>

In [19]:
col = 'ctl az'
fig = plt.figure(figsize=(8,6), dpi=100)
ax1 = sns.distplot(clean_data[col], bins=100, kde=False, fit=stats.norm, legend = True)
ax1.legend()
ax1.set_title('HEROES/SAS PYAS-F $x_{mean}$ = ' + str(clean_data[col].mean()) + ' $\sigma_x$ = ' + str(clean_data[col].std())[0:6])
ax1.set_xlabel(col + ' [arcsec]')
plt.show()



In [20]:
col = 'ctl el'
fig = plt.figure(figsize=(8,6), dpi=100)
ax1 = sns.distplot(clean_data[col], bins=100, kde=False, fit=stats.norm, legend = True)
ax1.legend()
ax1.set_title('HEROES/SAS PYAS-F $x_{mean}$ = ' + str(clean_data[col].mean()) + ' $\sigma_x$ = ' + str(clean_data[col].std())[0:6])
ax1.set_xlabel(col + ' [arcsec]')
plt.show()



In [21]:
col = 'offset x'
fig = plt.figure(figsize=(8,6), dpi=100)
ax1 = sns.distplot(clean_data[col], bins=100, kde=False, fit=stats.norm, legend = True)
ax1.legend()
ax1.set_title('HEROES/SAS PYAS-F $x_{mean}$ = ' + str(clean_data[col].mean()) + ' $\sigma_x$ = ' + str(clean_data[col].std())[0:6])
ax1.set_xlabel(col + ' [arcsec]')
plt.show()


The mean should likely be zero as this is the offset. It's not clear to me where a dc offset would come from.


In [22]:
fig = plt.figure(figsize=(8,6), dpi=100)
clean_data[col].plot()


Out[22]:
<matplotlib.axes.AxesSubplot at 0x103abb090>

In [ ]:
fig = plt.figure(figsize=(8,6), dpi=100)
clean_data[col][10000:12000].plot()

In [23]:
col = 'offset y'
fig = plt.figure(figsize=(8,6), dpi=100)
ax1 = sns.distplot(clean_data[col], bins=100, kde=False, fit=stats.norm, legend = True)
ax1.legend()
ax1.set_title('HEROES/SAS PYAS-F $x_{mean}$ = ' + str(clean_data[col].mean()) + ' $\sigma_x$ = ' + str(clean_data[col].std())[0:6])
ax1.set_xlabel(col + ' [arcsec]')
plt.show()


The norm fit is not great here. Constraining the data may help.


In [24]:
col = 'offset r'
fig = plt.figure(figsize=(8,6), dpi=100)
ax1 = sns.distplot(clean_data[col], bins=100, kde=True, legend = True)
ax1.legend()
ax1.set_title('HEROES/SAS PYAS-F $x_{mean}$ = ' + str(clean_data[col].mean()) + ' $\sigma_x$ = ' + str(clean_data[col].std())[0:6])
ax1.set_xlabel(col + ' [arcsec]')
plt.show()


Now lets plot the pointing accuracy over the entire solar observation as an offset from the target


In [56]:
kde2d = stats.gaussian_kde([clean_data['offset x'].values, clean_data['offset y'].values])
density = kde2d(heatmap)


---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-56-89e3f4738f44> in <module>()
      1 kde2d = stats.gaussian_kde([clean_data['offset x'].values, clean_data['offset y'].values])
----> 2 density = kde2d(heatmap)

/Users/schriste/anaconda/lib/python2.7/site-packages/scipy/stats/kde.pyc in evaluate(self, points)
    219                 msg = "points have dimension %s, dataset has dimension %s" % (d,
    220                     self.d)
--> 221                 raise ValueError(msg)
    222 
    223         result = zeros((m,), dtype=np.float)

ValueError: points have dimension 200, dataset has dimension 2
ERROR: ValueError: points have dimension 200, dataset has dimension 2 [scipy.stats.kde]

In [63]:
heatmap, xedges, yedges = np.histogram2d(clean_data['offset x'].values, clean_data['offset y'].values, bins=200, normed = True)

In [73]:
cumsum = np.cumsum(heatmap, axis=0)

In [74]:
plt.figure()
plt.imshow(cumsum)


Out[74]:
<matplotlib.image.AxesImage at 0x10e2ae5d0>

In [58]:
extent1 = [yedges[0], yedges[-1], xedges[-1], xedges[0]]
fig, ax = plt.subplots(figsize=(8,6), dpi=100)
#cax = plt.hist2d(clean_data['offset x'].values, clean_data['offset y'].values, bins=100, cmap = "PuRd", normed=True)
#ax.contour(kde2d, extent = extent)
cax = plt.imshow(heatmap, extent = extent1, origin = 'upper', hold = 'on', cmap="PuRd")
cax.set_interpolation('bilinear')
cnt = plt.contour(heatmap, extent = extent1, origin = 'upper')
ax.set_title('HEROES/SAS PYAS-F')
ax.set_xlabel('Heliospheric Offset X [arcsec]')
ax.set_ylabel('Heliospheric Offset Y [arcsec]')
ax.set_xlim(-100,100)
ax.set_ylim(-100,100)
plt.colorbar()
plt.show()



In [59]:
from sunpy.map import Map
file = '/Users/schriste/Downloads/aia.lev1.193A_2013-09-21T16_00_06.84Z.image_lev1.fits'
map = Map(file)
fov = 9*60
target = np.array([-794., 102])
corner = target - fov/2.0
xrange = corner[0] + np.array([0, fov])
yrange = corner[1] + np.array([0, fov])
smap = map.submap(xrange, yrange)
 
heatmap, xedges, yedges = np.histogram2d(clean_data['pointing y'].values, clean_data['pointing x'].values, bins=200)
extent = [yedges[0], yedges[-1], xedges[-1], xedges[0]]


WARNING: Converting Quantity object in units 'kg3 m N' to a Numpy array [astropy.units.quantity]

In [60]:
heatmap.max(), heatmap.min()


Out[60]:
(551.0, 0.0)

In [61]:
masked_data = np.ma.masked_where(heatmap < 10, heatmap)

In [62]:
fig, ax = plt.subplots(figsize=(15,10), dpi=100)
cax = smap.plot()
smap.draw_limb()
ax.plot(target[0], target[1], "w+")
ax.imshow(masked_data, extent = extent, alpha = 0.5, cmap = plt.get_cmap('jet_r'))
ax.set_xbound(xrange[0], xrange[1])
ax.set_ybound(yrange[0], yrange[1])
plt.show()



In [ ]: